{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Otto商品分类——RBF 核SVM\n",
    "\n",
    "我们以Kaggle 2015年举办的Otto Group Product Classification Challenge竞赛数据为例，分别调用\n",
    "缺省参数SVC、\n",
    "SVC + GridSearchCV进行参数调优。\n",
    "\n",
    "Otto数据集是著名电商Otto提供的一个多类商品分类问题，类别数=9. 每个样本有93维数值型特征（整数，表示某种事件发生的次数，已经进行过脱敏处理）。 竞赛官网：https://www.kaggle.com/c/otto-group-product-classification-challenge/data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#竞赛的评价指标为logloss\n",
    "#from sklearn.metrics import log_loss  \n",
    "\n",
    "#SVM虽然也支持输出各类的概率，但这需要额外的计算费用，且得到的概率也不保证是合法的概率，\n",
    "#所以在这个例子中我们用正确率accuracy_score作为模型选择的度量，最后在最佳超参数情况下再训练模型，得到概率表示\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pca_0</th>\n",
       "      <th>pca_1</th>\n",
       "      <th>pca_2</th>\n",
       "      <th>pca_3</th>\n",
       "      <th>pca_4</th>\n",
       "      <th>pca_5</th>\n",
       "      <th>pca_6</th>\n",
       "      <th>pca_7</th>\n",
       "      <th>pca_8</th>\n",
       "      <th>...</th>\n",
       "      <th>pca_25</th>\n",
       "      <th>pca_26</th>\n",
       "      <th>pca_27</th>\n",
       "      <th>pca_28</th>\n",
       "      <th>pca_29</th>\n",
       "      <th>pca_30</th>\n",
       "      <th>pca_31</th>\n",
       "      <th>pca_32</th>\n",
       "      <th>pca_33</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>44642</td>\n",
       "      <td>-2.401485</td>\n",
       "      <td>4.392327</td>\n",
       "      <td>0.520809</td>\n",
       "      <td>-2.201817</td>\n",
       "      <td>-1.233103</td>\n",
       "      <td>-0.482438</td>\n",
       "      <td>0.371232</td>\n",
       "      <td>2.679550</td>\n",
       "      <td>-0.693654</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.725258</td>\n",
       "      <td>0.215070</td>\n",
       "      <td>0.259147</td>\n",
       "      <td>0.616864</td>\n",
       "      <td>0.221898</td>\n",
       "      <td>0.028583</td>\n",
       "      <td>0.618058</td>\n",
       "      <td>0.730880</td>\n",
       "      <td>0.236966</td>\n",
       "      <td>Class_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34886</td>\n",
       "      <td>-1.613945</td>\n",
       "      <td>6.905888</td>\n",
       "      <td>-0.684274</td>\n",
       "      <td>-6.444908</td>\n",
       "      <td>-3.673110</td>\n",
       "      <td>7.966786</td>\n",
       "      <td>29.666846</td>\n",
       "      <td>19.479659</td>\n",
       "      <td>11.027421</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.307331</td>\n",
       "      <td>-0.631211</td>\n",
       "      <td>-0.869630</td>\n",
       "      <td>0.638846</td>\n",
       "      <td>0.124559</td>\n",
       "      <td>1.460256</td>\n",
       "      <td>1.675033</td>\n",
       "      <td>0.120857</td>\n",
       "      <td>0.192366</td>\n",
       "      <td>Class_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>44807</td>\n",
       "      <td>-2.917818</td>\n",
       "      <td>13.187745</td>\n",
       "      <td>-0.953303</td>\n",
       "      <td>-3.309427</td>\n",
       "      <td>-0.126315</td>\n",
       "      <td>0.449860</td>\n",
       "      <td>-1.212859</td>\n",
       "      <td>-0.921445</td>\n",
       "      <td>-0.654326</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.023734</td>\n",
       "      <td>0.175676</td>\n",
       "      <td>-0.922382</td>\n",
       "      <td>0.176985</td>\n",
       "      <td>-0.817531</td>\n",
       "      <td>0.840582</td>\n",
       "      <td>0.059230</td>\n",
       "      <td>3.272305</td>\n",
       "      <td>-1.957479</td>\n",
       "      <td>Class_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19017</td>\n",
       "      <td>-4.250615</td>\n",
       "      <td>-7.726496</td>\n",
       "      <td>0.474893</td>\n",
       "      <td>9.366311</td>\n",
       "      <td>-1.566268</td>\n",
       "      <td>13.076900</td>\n",
       "      <td>2.782658</td>\n",
       "      <td>-6.785141</td>\n",
       "      <td>0.338254</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.372314</td>\n",
       "      <td>3.791879</td>\n",
       "      <td>2.771023</td>\n",
       "      <td>-1.568888</td>\n",
       "      <td>2.397054</td>\n",
       "      <td>-0.444911</td>\n",
       "      <td>-0.000162</td>\n",
       "      <td>1.950526</td>\n",
       "      <td>-1.835864</td>\n",
       "      <td>Class_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>25876</td>\n",
       "      <td>-1.112903</td>\n",
       "      <td>-0.338095</td>\n",
       "      <td>0.491536</td>\n",
       "      <td>-0.069408</td>\n",
       "      <td>-1.208544</td>\n",
       "      <td>-2.191797</td>\n",
       "      <td>-1.129590</td>\n",
       "      <td>2.516341</td>\n",
       "      <td>-1.490161</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.239775</td>\n",
       "      <td>-0.920964</td>\n",
       "      <td>0.184961</td>\n",
       "      <td>-1.144437</td>\n",
       "      <td>0.133968</td>\n",
       "      <td>-1.236665</td>\n",
       "      <td>-1.801884</td>\n",
       "      <td>-0.609749</td>\n",
       "      <td>-1.325505</td>\n",
       "      <td>Class_3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      id     pca_0      pca_1     pca_2     pca_3     pca_4      pca_5  \\\n",
       "0  44642 -2.401485   4.392327  0.520809 -2.201817 -1.233103  -0.482438   \n",
       "1  34886 -1.613945   6.905888 -0.684274 -6.444908 -3.673110   7.966786   \n",
       "2  44807 -2.917818  13.187745 -0.953303 -3.309427 -0.126315   0.449860   \n",
       "3  19017 -4.250615  -7.726496  0.474893  9.366311 -1.566268  13.076900   \n",
       "4  25876 -1.112903  -0.338095  0.491536 -0.069408 -1.208544  -2.191797   \n",
       "\n",
       "       pca_6      pca_7      pca_8  ...    pca_25    pca_26    pca_27  \\\n",
       "0   0.371232   2.679550  -0.693654  ... -0.725258  0.215070  0.259147   \n",
       "1  29.666846  19.479659  11.027421  ... -0.307331 -0.631211 -0.869630   \n",
       "2  -1.212859  -0.921445  -0.654326  ... -0.023734  0.175676 -0.922382   \n",
       "3   2.782658  -6.785141   0.338254  ... -0.372314  3.791879  2.771023   \n",
       "4  -1.129590   2.516341  -1.490161  ... -0.239775 -0.920964  0.184961   \n",
       "\n",
       "     pca_28    pca_29    pca_30    pca_31    pca_32    pca_33   target  \n",
       "0  0.616864  0.221898  0.028583  0.618058  0.730880  0.236966  Class_6  \n",
       "1  0.638846  0.124559  1.460256  1.675033  0.120857  0.192366  Class_6  \n",
       "2  0.176985 -0.817531  0.840582  0.059230  3.272305 -1.957479  Class_6  \n",
       "3 -1.568888  2.397054 -0.444911 -0.000162  1.950526 -1.835864  Class_3  \n",
       "4 -1.144437  0.133968 -1.236665 -1.801884 -0.609749 -1.325505  Class_3  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "\n",
    "#原始特征 + tf_idf特征对线性SVM训练还是很快，RBF核已慢得不行\n",
    "# RBF核只用tf_idf特征\n",
    "train = pd.read_csv(dpath +\"Otto_FE_train_PCA_random10000.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10000 entries, 0 to 9999\n",
      "Data columns (total 36 columns):\n",
      "id        10000 non-null int64\n",
      "pca_0     10000 non-null float64\n",
      "pca_1     10000 non-null float64\n",
      "pca_2     10000 non-null float64\n",
      "pca_3     10000 non-null float64\n",
      "pca_4     10000 non-null float64\n",
      "pca_5     10000 non-null float64\n",
      "pca_6     10000 non-null float64\n",
      "pca_7     10000 non-null float64\n",
      "pca_8     10000 non-null float64\n",
      "pca_9     10000 non-null float64\n",
      "pca_10    10000 non-null float64\n",
      "pca_11    10000 non-null float64\n",
      "pca_12    10000 non-null float64\n",
      "pca_13    10000 non-null float64\n",
      "pca_14    10000 non-null float64\n",
      "pca_15    10000 non-null float64\n",
      "pca_16    10000 non-null float64\n",
      "pca_17    10000 non-null float64\n",
      "pca_18    10000 non-null float64\n",
      "pca_19    10000 non-null float64\n",
      "pca_20    10000 non-null float64\n",
      "pca_21    10000 non-null float64\n",
      "pca_22    10000 non-null float64\n",
      "pca_23    10000 non-null float64\n",
      "pca_24    10000 non-null float64\n",
      "pca_25    10000 non-null float64\n",
      "pca_26    10000 non-null float64\n",
      "pca_27    10000 non-null float64\n",
      "pca_28    10000 non-null float64\n",
      "pca_29    10000 non-null float64\n",
      "pca_30    10000 non-null float64\n",
      "pca_31    10000 non-null float64\n",
      "pca_32    10000 non-null float64\n",
      "pca_33    10000 non-null float64\n",
      "target    10000 non-null object\n",
      "dtypes: float64(34), int64(1), object(1)\n",
      "memory usage: 2.7+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将类别字符串变成数字\n",
    "# drop ids and get labels\n",
    "y_train = train['target']   #形式为Class_x\n",
    "X_train = train.drop([\"id\", \"target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_split.py:2179: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 训练样本6w+，交叉验证太慢，用train_test_split估计模型性能\n",
    "# SVM对大样本数据集支持不太好\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train_part, X_val, y_train_part, y_val = train_test_split(X_train, y_train, train_size = 0.8,random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8000, 34)\n"
     ]
    }
   ],
   "source": [
    "print (X_train_part.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RBF核SVM正则参数调优\n",
    "\n",
    "RBF核是SVM最常用的核函数。\n",
    "RBF核SVM 的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和核函数的宽度gamma\n",
    "C越小，决策边界越平滑； \n",
    "gamma越小，决策边界越平滑。\n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_grid_point_RBF(C, gamma, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC3 =  SVC( C = C, kernel='rbf', gamma = gamma)\n",
    "    SVC3 = SVC3.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC3.score(X_val, y_val)\n",
    "    \n",
    "    print(\"C= {} and gamma = {}: accuracy= {} \" .format(C, gamma, accuracy))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0.]\n",
      " [0. 0. 0.]\n",
      " [0. 0. 0.]\n",
      " [0. 0. 0.]\n",
      " [0. 0. 0.]]\n",
      "[0.0001 0.001  0.01  ]\n"
     ]
    }
   ],
   "source": [
    "accuracy_s = np.matrix(np.zeros(shape=(5, 3)), float)\n",
    "print(accuracy_s)\n",
    "gamma_s = np.logspace(-4, -2, 3)  \n",
    "print(gamma_s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 0.1 and gamma = 0.1: accuracy= 0.3875 \n",
      "C= 0.1 and gamma = 1.0: accuracy= 0.279 \n",
      "C= 0.1 and gamma = 10.0: accuracy= 0.278 \n"
     ]
    }
   ],
   "source": [
    "oneC = 0.1\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[0,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 1 and gamma = 0.1: accuracy= 0.553 \n",
      "C= 1 and gamma = 1.0: accuracy= 0.298 \n",
      "C= 1 and gamma = 10.0: accuracy= 0.2795 \n"
     ]
    }
   ],
   "source": [
    "oneC = 1\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[1,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 10 and gamma = 0.1: accuracy= 0.564 \n",
      "C= 10 and gamma = 1.0: accuracy= 0.3015 \n",
      "C= 10 and gamma = 10.0: accuracy= 0.28 \n"
     ]
    }
   ],
   "source": [
    "oneC = 10\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[2,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 100 and gamma = 0.1: accuracy= 0.5615 \n",
      "C= 100 and gamma = 1.0: accuracy= 0.302 \n",
      "C= 100 and gamma = 10.0: accuracy= 0.28 \n"
     ]
    }
   ],
   "source": [
    "oneC = 100\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[3,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 1000 and gamma = 0.0001: accuracy= 0.747 \n",
      "C= 1000 and gamma = 0.001: accuracy= 0.746 \n",
      "C= 1000 and gamma = 0.01: accuracy= 0.7045 \n"
     ]
    }
   ],
   "source": [
    "oneC = 1000\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[4,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#需要调优的参数\n",
    "#C_s = np.logspace(-1, 3, 5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "#gamma_s = np.logspace(-1, 1, 3)    \n",
    "\n",
    "#accuracy_s = []\n",
    "#for i, oneC in enumerate(C_s):\n",
    "    #for j, gamma in enumerate(gamma_s):\n",
    "        #tmp = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)\n",
    "        #accuracy_s[i,j] = tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上述结果会发现，gamma参数非常重要(当gamma=0.01或gamma=100时性能很差),非线性模型比线性模型性能好（注意我们这里只用了tfidf特征）。\n",
    "但速度慢了不是一点半点(sklearn建议核方法SVM样本数不超过10000)\n",
    "可以考虑将训练样本分为多个子集，每个子集训练一个RBF核SVM模型，最后多个模型融合的结果的到最终模型（训练速度加快，但测试可能更慢）\n",
    "\n",
    "C= 0.1 and gamma = 0.1: accuracy= 0.738122171946 \n",
    "C= 0.1 and gamma = 1.0: accuracy= 0.763897866839 \n",
    "C= 0.1 and gamma = 10.0: accuracy= 0.549208144796 \n",
    "\n",
    "C= 1 and gamma = 0.1: accuracy= 0.761716224952 \n",
    "C= 1 and gamma = 1.0: accuracy= 0.800985778927 \n",
    "C= 1 and gamma = 10.0: accuracy= 0.71630575307\n",
    "\n",
    "C= 10 and gamma = 0.1: accuracy= 0.78603749192 \n",
    "C= 10 and gamma = 1.0: accuracy= 0.817065287654 \n",
    "C= 10 and gamma = 10.0: accuracy= 0.727052359405 \n",
    "\n",
    "C= 100 and gamma = 0.1: accuracy= 0.805914673562 \n",
    "#### C= 100 and gamma = 1.0: accuracy= 0.807449903038 \n",
    "C= 100 and gamma = 10.0: accuracy= 0.724951519069\n",
    "\n",
    "C= 1000 and gamma = 0.1: accuracy= 0.805914673562 \n",
    "C= 1000 and gamma = 1.0: accuracy= 0.794521654816 \n",
    "C= 1000 and gamma = 10.0: accuracy= 0.724062702004 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VPW5+PHPk33fyAJJSMImkEQSICwCXlCroiBoL71o68V9uS7XpXrVlttFcWuLrb+6VGup2tri1SpSa2txb6sgARFZlH1JAANkJSHrfH9/nJPJJCTMJGRysjzv12temTnzPTPPTJJ55ruc54gxBqWUUupkApwOQCmlVO+nyUIppZRXmiyUUkp5pclCKaWUV5oslFJKeaXJQimllFeaLJRSSnmlyUIppZRXmiyUUkp5FeR0AN0lMTHRZGVlOR2GUkr1KevWrTtijEny1q7fJIusrCwKCwudDkMppfoUEdnrSzsdhlJKKeWVJgullFJeabJQSinllSYLpZRSXmmyUEop5ZUmC6WUUl5pslBKKeVVvznOQqlepakByvbCkW1wdDsgEJUCUcn2zxQIj4cA/b6m+gZNFkqdiuPlcHSHlRSObIMj262fpbvA1XjyfQOCIDIZopJOTCSe1yOTIDQaRHrmNSnVDk0WSnnjckHFfquH0JwMmn8e+7qlXUAQJIyAxNNgzFzrZ+JpkDgSEDhWYrU/9rXHdY9th76wbpumE2MICvdIIG2Tisf2yGQIDuuxt0YNHJoslGpWX+PRS/BICkd3QOPxlnZhcVYSGHkuJI5qSQrxmRAY3PHjh8XYieMkXC44XtpBUrGvH90Jez+22rX7PLGtk0lkcvu9lshECAjs/PukBiRNFmpgMcb6wHUPG3kkh4p9Hg3F+vBPPA2Gz2ydFCIG+W9IKCDA+hCPTISUnJO3bayH6sPW62n+2TbBHPjM+ll/7MT9JQAiEk/snZyQVJKs+RUdBhvQNFmo/qmxHsp2nziXcGQ71FW2tAuOtL7tZ0yBxP9sSQoJI3r/cE5QCMSmWRdv6qtb907aJpXqEuu9OfY1NNWfuH9gSJseSnL7w2BRKRAS2f2vVTlOk4Xq22pKPRLBtpZhpNLdrcf+o1OtRDBuod1DsJNCTOrA+MYcEgkJw6zLyRgDteXtzKl4JJqKIiheZ/VmMO08V9SJ8ygnJBi7xxIU4peXq7qfJgvV+7maoHxf66TQfL3mSEu7wBAYNBKSsyH7Yo+kMMpaTaS8E7GGnMLjIWn0yds2NULNUXsYrKT9BFOyFY59ALUV7T9GeEKb3omdTGLSIC7DukQm6xLjXkCTheo96o61WXHUPMG8E5rqWtpFDLJXHF3oseJoFMRl6oRtTwoMgugU6+JNQ609r9LeMJg951K0FlP1NeK5mABwBYRQG5lmXaLSqY20L1Fp1Eam0xCWODB6hycRHRbEhIx4vz6HJgvVs4yBygPtJ4XK4pZ2EgDxw+xVR+e0JIVBoyBykHPxq64JDoO4odbFVlnbwJYDlWwqrmDLgUo2V1ayo6aKcFcNg6WUdDlMuhwhXQ4ztKGE9IoDpMvnpEpVq4euNcEUm0SKTJLHJZH9Jpkik8QRYoD+nUzyh8ax4ubpfn0OvyYLEZkNPA4EAs8ZYx5pc38G8AIQZ7e51xjzln3ffcA1QBPw38aYt/0Zq+pmDbXWgWmtJpftOQXPlTkh0VavIOvM1iuOEoZBUKhz8atudbiqjk0H7KRwoIJNxZXsK61x358cHUpOagzn5aSQkxpDUvSJiwsagT3AvoZqQo4VE3KsiNBjRYQcKyLhWBGDq/Yz/dh6gurKWu3nCgylLiqdevvSfL0u2vrZ2A96JpGh/u9R+y1ZiEgg8CRwLlAErBWRlcaYLR7NFgP/Z4x5WkSygbeALPv6pUAOkAq8IyKnGdPe0UrKMcZYY9ZtVxsd2Qble8G4WtrGDrWSQcbl1rxCc1KIHtzn/1FVC2MMRWXH2Xyggs0HKtls9xxKqlqGETMSIshNi2HhpKFkp8aQkxpDcjvJoWPxQDowpf2766qgfL81z1W+j4DyvYSX7yO8fB/s++LE41OCwlvmR1pdMu05k76fTLqDP3sWk4EdxphdACKyHJgPeCYLA8TY12OBA/b1+cByY0wdsFtEdtiP94kf41XeuFywfzVseg0ObbSSwnGPb3FBYVYiSM2Hcf/RMpcwaKQup+yHmlyGXYeP2UnB6i1sOVhJxfEGAAIERiVHM2NkItmpMeSmxZKdGkNM2EkOXOwOodGQkm1d2lNX5U4kLZe91s/iwtZ/09CSTOIz208o/jzuphfxZ7JIA/Z73C7ixK8CPwL+LiK3ApHANzz2Xd1mXx8Wkyu/OPwVbHwZNr5iHbgWHAGpE05ccRQ7VCeY+6m6xia2HTpmJQW71/DlwSqON1id/ZCgAMYOjubC04eQYyeGMYOjCQvuhX8PodHWAY8dHfRYW2mVdynfZxWD9Ewm+z+1lhZ7Co44sTfSKpkk9Itk4vQE92XA88aYpSJyBvA7Ecn1dWcRuR64HiAjI8NPIQ5QVYdg05+sJHHwc2vCefhZcPZiGDMHQqOcjlD5ybG6RrYetIaPmoeStn9dRaPLOqYiKjSI7NQYLp08lNzUWHLSYhiRFEVwYD9Z3hoWA2EnSyYVrYa5WpLJXti/5sRlwsGRJ/ZI3L2UzD5zdLw/k0UxMNTjdrq9zdM1wGwAY8wnIhIGJPq4L8aYZ4FnAQoKCto5Okh1St0x+PJNK0Hs+sCacxiSD+c/DLn/7tsSSdWnlFbXu4eQNtsT0LuPVmPs/6ZBkSHkpMUya3SSlRhSY8hIiCAgoPd/uPlNWCwMjoXBHXyvPV7e0jNpm1D2rYa6NskkJKqDOZPelUz8mSzWAqNEZBjWB/2lwLfbtNkHnAM8LyJjgTDgMLAS+IOIPIY1wT0K+NSPsQ5cTY1WYtj4spUoGmogNgNm3GnNO3g7MEv1CcYYDlbUtuotbD5QwcGKWnebtLhwclJjuHh8GjmpMeSkxpISE4r0gg+qPiU8zroMPr39+4+Xt5NI7Mvej1uXowFrxWCHySSjx5KJ35KFMaZRRG4B3sZaFrvMGLNZRO4HCo0xK4HvAr8WkTuwJruvNMYYYLOI/B/WZHgjcLOuhOpGxlgF5jb+H2x61TogKizOKoUxbiEMnaJHzPZhLpdh99Fqd0LYbPcaymqsiWcRGJ4YyeRhCe6kkJMaQ1yElt7oEc3JZMi49u8/Xt4yR9L2suefUN/6OBNCY2DYv8GlL/k1bDGmf4zeFBQUmMLCQqfD6N3K9liT1Btftg6KCwyB02ZbCWLUuXpcQx9U3+hie0mVlRjsXsPWg5VU11vfrYIDhdNSot1zCzmpsYwdEk1EiNPTlapLmmt3eSaQsr3WJPqse7v0kCKyzhhT4K2d/sX0dzWlsPl1qxex315gljkdpt0C2fOtLqzqE2rqG9l6sIotzXMMByvYdugY9U3W8SwRIYFkD4lhwcR0q7eQFsOo5GhCgrSX2G941u4aktejT63Joj9qqIXtb8PnL8P2v4OrARJHwzk/gNO/ZY1zql6toqah1TLVzQcq2XX4GPaCJOIjgslJjeWq6VnkpFnDSFmDIgkcyBPPyq80WfQXLhfs+9gaYtr8hrXiIioFptxgTVQPHtcrVlSo1owxlFTVeUw8W72G4vKWYnpDYsPISY1hjn0MQ05aLKmxYTrxrHqUJou+rmRrywFzlUXWmu6xF0HeQhg2Uw+S60VcLsO+0pqWpHCgki0HKjhyrOVkQ8MSI8nPiOPyqZn25HMMg6J0Lkk5T5NFX1R50FrFtPFlOPQFSCCMOBu+8SOrbLeW1ugVjh6rY+2eMgr3lLKxuIKtByqpqmsEIChAGJkcxazRye4VSWOHRBPt71IYSnWRJou+oq4KttoHzO3+0DpgLnUCzH4Ucr9pnTBGOcYYq9ewdk8Za3eXsnZvKbsOVwNWKYzsITHMH59KTmosuamxjEqJ6p2lMJTqgCaL3qypAXa+bx8w9xdoPG4d0XnmXXahvlFORzhgNbkMWw9WsnZPKYV7yli7p9RdWTUmLIhJWQl8a+JQJmXFc3p6LKFBmhhU36bJorcxBorXWwli05+s04aGxUH+ZTDuUhg6WSeqHVDb0MRn+8op3FPKp3tK+WxfOcfsIaW0uHDOGDGIgqwEJmclMCo5amCXw1D9kiaL3qJ0V8sBc6U7ITAURtsHzI08V09s38PKqusp3FvmTg6biitoaLLWrY5Oiebi8alMykqgICuBtLhwh6NVyv80WTipptSu7Pp/UGSXvso6E2bcDmPnWSUBlN81n7Bn7Z5S94T09hLrbH4hgQGMS4/lmhnDmZQVz8TMeC2LoQYkTRY9reE4bPublSC2/x1cjZA01lrJlLug1TmKlX80uQxfHaqicG8pn+625hwOVVoF9aJDg5iYFc/F49OYlJXAuPRYnYhWCk0WPcPlgr3/tIaYtqy0qkpGD4Gp/2UNM6Xk6jyEH9U2NLGxqMLuOZSybm8ZVbXWfMPgmDAmDUtgUlY8BZkJjB4crUdBK9UOTRb+9PVmK0F88SpUFlt168fOs1YyDfs3PWDOTypqGli3r5RPd9vHOBRVuOsnjUqOYu64VCZlxTMpK4H0+HA9ElopH2iy6G6VB+CLV6xhpq83WQfMjfwGnHs/jL4QQiKcjrDfOVB+3N1rWLu7jK++tko4BwUIp6fHcuX0LCZlJTAxM56ESJ1vUKorNFl0h9pK2Ppn+4C5jwADaQVwwU8h5xKISnI6wn7D5TJsLznmTg6Fe8rcdZSiQoOYkBnP3HFDKMhKIH9oHOEh2ntTqjtosuiqpgbY8a6VIL56CxprIX4YzLzHGmYaNMLpCPuFusYmNhVXuIeUCveWUXHcOolPUnQok7MSuPbMYUzKSmDM4GiC+st5oJXqZTRZdIYxUFRoV3Z9DWqOQngCjL/cmqhOn6QT1aeosraBdfbxDWt3l7GhqJz6Rmu+YXhSJLNzBrsnpDMSInS+QakeosnCF0d32vMQL1sHzwWFwegLrAQx4hw9YO4UHKqobZlv2FPGl4cqMQYCA4TctFgWTc2kICuBgqx4ErX6qlKO0WTRkeoj9hnmXoaitYDAsDPhzO9aJcDDYp2OsM8xxrDz8DH3kNLavaXsL7XmGyJCApmQEc9t54xiclYC+RlxeupPpXoR/W/01HDcmn/Y+H+w4x3rgLnkHGslU+4CiE1zOsI+pb7RxaYDFVZisI+MLqux5hsSo0IoyEzgymnDmJQVT/aQGJ1vUKoX02ThaoI9/7ASxJaVUF8F0akw9SZrmGlwrtMR9hnH6hpZ71FPacP+cmobrPmGYYmRfGNsCpOyEpg0LIGsQTrfoFRfosmiYj+8OB9CoiF7vrWSKWuGHjDng5LKWuv8DXtKKdxbypYDlbgMBAjkpMZy2eQMJmclMDErnuToMKfDVUqdAk0W8Vlw+WuQOQ2CtXpoR1wua77BqsRaRuHeUvYerQEgLDiA8UPjueXsUUzKimd8RjxRofqnpVR/ov/RACPPcTqCXqemvpHP91ewbq9VS2n9vnL38Q0JkSEUZMZz+ZRMJg1LICc1hmCdb1CqX9NkoQBrCWuhnRjW7S1jy4FKGl3W+RtGJUdxQe5gJmbGU5Cl8w1KDUSaLAagxiYXXx6qcieGdXtbSmaEBQeQPzSOG2YOpyAzgfEZcXr+BqWUJouBoLK2gc/2lduJoZQN+8qprm8CICUmlILMBK6ZMYyJmfFk65CSUqodmiz6GWMM+0uPs26fVWRv3V6rCquxVymNGRzDv09MZ2Kmdda3tDgt0a2U8s6vyUJEZgOPA4HAc8aYR9rc/3PgLPtmBJBsjImz72sCvrDv22eMmefPWPuq+kYXmw9U2PWUyli3r4zDVXWAVYV1fEYcF+QOYWJmPPkZcbpKSSnVJX775BCRQOBJ4FygCFgrIiuNMVua2xhj7vBofysw3uMhjhtj8v0VX19VWl1vHfi2t4z1e8v4vKicOrvQ3tCEcGaMTGRCZjwFmfGclqJnfVNKdQ9/fs2cDOwwxuwCEJHlwHxgSwftLwN+6Md4+hyrllK1e/lq4d4ydh2uBiA4UMhJjeXyqZkU2ENKyTF64JtSyj/8mSzSgP0et4uAKe01FJFMYBjwnsfmMBEpBBqBR4wxK/wVaG9R29DE5/vLWbevjHX2kFK5XUspPiKYiZnxLJiYTkFmAuPSYwkL1qPMlVI9o7cMYF8KvGqMafLYlmmMKRaR4cB7IvKFMWan504icj1wPUBGRkbPRdtNSiprKbSXrhbuLWNzcYX72IYRSZGcl51CQWYCEzLjGZEUqRPRSinH+DNZFANDPW6n29vacylws+cGY0yx/XOXiHyANZ+xs02bZ4FnAQoKCky3RO0nTS7DV4eq7F5DKev2lbnLc4cGBZCXHsd1/zacgkyrXIaeK1op1Zv4M1msBUaJyDCsJHEp8O22jURkDBAPfOKxLR6oMcbUiUgiMB34iR9j7XbH6hr5bF/LQW+f7SvnWF0jYJ0OtCAznivOyGJiZjw5qbGEBOmxDUqp3stvycIY0ygitwBvYy2dXWaM2Swi9wOFxpiVdtNLgeXGGM+ewVjgGRFxAQFYcxYdTYw7zhhDcfnxluWre60zvrmMdZbV0SnRXDw+1SqXkZlAerwe26CU6luk9Wd031VQUGAKCwt75LkamlxsOVDpXr5auLeUryutYxsiQwIZnxHvPuhtfEYc0WHBPRKXUkp1loisM8YUeGvXWya4e7XymnrW20NKhXusYxuaT+qTFhfO1OGD3MlhdEq0nvFNKdXvaLJowxjD7iPVHr2GMnaUHAMgKEDISY3hsskZFGQmMDEznsGxemyDUqr/G/DJor7RxedF5e5ew/p9ZZRW1wMQGx7MhIw4LhmfxsTMePLS4wgP0WMblFIDz4BPFqXV9XzrV9ZCrOGJkZw9Jtl9RPSIpCgCtFyGUkppshgcG8Zvr5rEuLRYBkWFOh2OUkr1SgM+WQCcNTrZ6RCUUqpX02U7SimlvNJkoZRSyitNFkoppbzSZKGUUsorneBWys9qGmpocDUQHBBMUEAQQQFBBIh+T1N9iyYLpbqowdXA0eNHKakpoaSmhK9rvuZwzWH37ZLj1s/qhuoT9g2QgFbJI0isn57bmq+3atf2trTftu0+7T6OBBMcGOx+DG/7tf2pCW9g0WShVBvGGCrrK1s+9Nte7CRw9PhRDK0LcQYFBJEUnkRyRDIj40YyLXUaSeFJhAaG0uhqpMHV0PLTNFrXm1que7bxvDS4GqhtrHXv19Bkt+lgv6ZW5xHzjwAJaElWHSSd9pJfqyTXzn7BAcEEBgQiCAESQIAEICIESqB13WO7+35ft4kQQEC37t+8LVACrese+3f0Ojra1purUWuyUANKXVMdJTUl7h6AZ2/g65qvOXzcul7XVHfCvnGhcSRHJJMckczYhLEkRVhJISUixZ0g4sPie8U3bpdxtZtEWiUi03hiAjtJsvLcr6GpoVXCa2+/9m57Jrz2EmVzuybThDEGl3GdkJD7s+Yk0l5yPNm2sQljefzsx/0amyYL1S+4jIvS2lJ3Ivi65mvr+nGP6zWHKa8rP2Hf0MBQdxLITcwlOdy6nhyZ7L6eFGH1DvqKAAkgJDCEkMC+f8bF5qThwtVyvfnS3jY7wbTd1pX92yatttua92+1zW7b7jZXU6vYmred6mtLj073++9Bk4Xq9Woaatw9AM8k4NkzOHz8MI2uxlb7CUJieCJJEUmkRaUxIXmCuweQEpHi7hnEhMT06u7/QNf87TkQLeLpJE0WyjGNrkaOHD/itTdwrOHYCftGBUe5v/FPGjzJuh6eREpEint7YngiQQH6J65Ud9D/JNXtTjpBfLzlersTxBJEUkQSSRFJrSaIPXsDKREpRARHOPTqlBqYNFmoTquoq2Bn+U7rg7/6xCRwuOYwtU21J+zXPEGcFJHEmIQx7fYGEsISesUEsVKqNU0WqlMOVR/i31f+O5X1le5tzRPESeFJ5A7KJXlosnvCuPnS1yaIlVKtabJQnfLIp49Q31TP42c9ztDooTpBrNQAoclC+eyD/R/w7r53uX3C7ZydcbbT4SilepAODiuf1DTU8PCahxkZN5JFOYucDkcp1cO0Z6F88szGZzhQfYDnZz9PcECw0+EopXqYTz0LEXlNROaI6DKVgWh72XZe3Pwil4y8hIkpE50ORynlAF8//J8Cvg1sF5FHRGS0H2NSvYjLuFiyeglRIVHcMfEOp8NRSjnEp2RhjHnHGPMdYAKwB3hHRD4WkatERMck+rE3drzB+pL13DnxTuLD4p0ORynlEJ+HlURkEHAlcC3wGfA4VvJY5ZfIlOPKastYum4pE5InMH/kfKfDUUo5yNc5i9eBfwARwEXGmHnGmJeNMbcCUSfZb7aIfCUiO0Tk3nbu/7mIbLAv20Sk3OO+K0Rku325ovMvTZ2qn6/7OdX11SyeuliPqlZqgPN1NdT/M8a8394dxpiC9raLSCDwJHAuUASsFZGVxpgtHvve4dH+VmC8fT0B+CFQABhgnb1vmY/xqlO07ut1vL7jda7OvZpR8aOcDkcp5TBfvy5mi0hc8w0RiReRm7zsMxnYYYzZZYypB5YDJxvLuAz4o339fGCVMabUThCrgNk+xqpOUUNTA0tWLyE1MpUbxt3gdDhKqV7A12RxnTHGPURkf4Bf52WfNGC/x+0ie9sJRCQTGAa819l9Vfd7ccuL7CjfwfemfE+ruyqlAN+TRaB4FP+xh5i68xRclwKvGtO5EweLyPUiUigihYcPH+7GcAau4mPF/OrzX3FOxjnMHDrT6XCUUr2Er8nib8DLInKOiJyDNVz0Ny/7FANDPW6n29vacyktQ1A+72uMedYYU2CMKUhKSvISjvLGGMNDax5CRLh38gnrEZRSA5ivyeIe4H3gv+zLu8D/eNlnLTBKRIaJSAhWQljZtpGIjAHigU88Nr8NnGfPjcQD59nblB+9t+89Pir6iJvzb2Zw5GCnw1FK9SI+rYYyxriAp+2LT4wxjSJyC9aHfCCwzBizWUTuBwqNMc2J41JguTHGeOxbKiIPYCUcgPuNMaW+PrfqvOqGah7+9GFGx4/mO2O/43Q4SqlexqdkISKjgIeBbCCsebsxZvjJ9jPGvAW81WbbD9rc/lEH+y4DlvkSnzp1T214ipKaEpbOWqrnrVZKncDXYajfYvUqGoGzgBeB3/srKNWzviz9kpe2vsSC0xaQl5TndDhKqV7I12QRbox5FxBjzF67NzDHf2GpnuIyLh745AFiQ2O5bcJtToejlOqlfB1vqLPLk2+35yGKOUmZD9V3vLrtVTYe2chDMx4iNjTW6XCUUr2Urz2L27DqQv03MBG4HNB6TX3ckeNH+MX6XzB58GTmDp/rdDhKqV7Ma8/CPgBvoTHmLuAYcJXfo1I9YmnhUmoba1k8dTEex1wqpdQJvPYs7KOqZ/RALKoHrTm4hjd3vcnVuVczLHaY0+EopXo5X+csPhORlcArQHXzRmPMa36JSvlVfVM9S1YvYWj0UK49/Vqnw1FK9QG+Josw4Chwtsc2A2iy6IOWbVrGnso9/OobvyIsKMz7DkqpAc/XI7h1nqKf2Fe5j19v/DWzs2YzPW260+EopfoIX4/g/i1WT6IVY8zV3R6R8htjDA+ueZCQwBDunnS30+EopfoQX4eh3vS4HgZcAhzo/nCUP729520+PvAx902+j+SIZKfDUUr1Ib4OQ/3J87aI/BH4p18iUn5RVV/Fo2sfJXtQNgtHL3Q6HKVUH9PVinGjAP1q2of88rNfcvT4UZ44+wkCAwKdDkcp1cf4OmdRRes5i0NY57hQfcDmI5tZ/uVyLh1zKTmJOU6Ho5Tqg3wdhor2dyDKP5pcTfz4kx8zKHwQt46/1elwlFJ9lE+1oUTkEhGJ9bgdJyIX+y8s1V2Wf7WcraVbuWfSPUSHaM5XSnWNr4UEf2iMqWi+YYwpB37on5BUdympKeGXn/2SaanTOD/rfKfDUUr1Yb4mi/ba6enUermfrv0pDU0NfH/K97VQoFLqlPiaLApF5DERGWFfHgPW+TMwdWr+Vfwv/rbnb1w37joyYjKcDkcp1cf5mixuBeqBl4HlQC1ws7+CUqemtrGWB9c8SFZMFlfn6kH2SqlT5+tqqGrgXj/HorrJc188x/6q/Tx33nOEBIY4HY5Sqh/wdTXUKhGJ87gdLyJv+y8s1VW7Knbxm02/Ye7wuUwZMsXpcJRS/YSvw1CJ9gooAIwxZegR3L2OMYYHVz9IeFA43y34rtPhKKX6EV+ThUtE3LOkIpJFO1VolbPe3PUmnx76lNsn3E5ieKLT4Sil+hFfl79+H/iniHwICHAmcL3folKdVlFXwc8Kf8a4pHEsOG2B0+EopfoZXye4/yYiBVgJ4jNgBXDcn4Gpznl8/eNU1FXw7LnPEiC+dhiVUso3vhYSvBa4DUgHNgBTgU9ofZpV5ZANJRt4ZdsrLMpexOiE0U6Ho5Tqh3z9CnobMAnYa4w5CxgPlJ98F9UTGl2NPLD6AVIiUrgp/yanw1FK9VO+JotaY0wtgIiEGmO+BLx+hRWR2SLylYjsEJF2j9MQkf8QkS0isllE/uCxvUlENtiXlT7GOeC8tPUltpVt477J9xEZHOl0OEqpfsrXCe4i+ziLFcAqESkD9p5sBxEJBJ4EzgWKgLUistIYs8WjzSjgPmC6MaZMRDyX4x43xuR34rUMOIeqD/HkhieZmT6TszN0RFAp5T++TnBfYl/9kYi8D8QCf/Oy22RghzFmF4CILAfmA1s82lwHPGkft4ExpqQTsQ94j3z6CMYY7ptynxYKVEr5VaeXzRhjPjTGrDTG1Htpmgbs97hdZG/zdBpwmoj8S0RWi8hsj/vCRKTQ3q7nzmjjg/0f8O6+d7kx70bSotq+rUop1b2cLjMehHU+71lYK60+EpHT7aPFM40xxSIyHHhPRL4wxuz03FlErsc+3iMjY+BUVq1pqOHhNQ8zMm4ki3IWOR2OUmoA8OeC/GJgqMftdHubpyJgpTGmwRizG9iGlTwwxhTbP3cBH2CtwGrFGPOsMabAGFON/zYaAAAbmklEQVSQlJTU/a+gl3pm4zMcqD7A4qmLCQ4IdjocpdQA4M9ksRYYJSLDRCQEuBRou6ppBVavAhFJxBqW2mUXKgz12D6d1nMdA9b2su28uPlFLhl5CRNTJjodjlJqgPDbMJQxplFEbgHeBgKBZcaYzSJyP1BojFlp33eeiGwBmoC7jTFHRWQa8IyIuLAS2iOeq6gGKpdxsWT1EqJCorhj4h1Oh6OUGkD8OmdhjHkLeKvNth94XDfAnfbFs83HwOn+jK0vemPHG6wvWc/90+4nPize6XCUUgOIFhHqI8pqy1i6bikTkicwf+R8p8NRSg0wmiz6iMfWPUZ1fTWLpy7WQoFKqR6nnzp9QOGhQlbsWMGinEWMih/ldDhKqQFIk0Uv19DUwJLVS0iNTOWGcTc4HY5SaoBy+qA85cULW15gZ8VOnjj7CSKCI5wORyk1QGnPohcrqirimc+f4ZyMc5g5dKbT4SilBjBNFr2UMYaHP30YEeHeye1Wd1dKqR6jyaKXem/fe3xU9BE359/M4MjBToejlBrgNFn0QtUN1Tz86cOMjh/Nd8Z+x+lwlFJKJ7h7o6c2PEVJTQlLZy0lKEB/RUop52nPopf5svRLXtr6EgtOW0BeUp7T4SilFKDJoldxGRcPfPIAsaGx3DbhNqfDUUopN00Wvcir215l45GN3FVwF7GhsU6Ho5RSbposeokjx4/wi/W/YPLgycwdPtfpcJRSqhVNFr3E0sKl1DbWsnjqYkTE6XCUUqoVTRa9wJqDa3hz15tcnXs1w2KHOR2OUkqdQJOFw+qb6lmyeglDo4dy7enXOh2OUkq1SxfxO2zZpmXsqdzDr77xK8KCwpwORyml2qU9Cwftq9zHrzf+mtlZs5meNt3pcJRSqkOaLBxijOHBNQ8SEhjC3ZPudjocpZQ6KU0WDnl7z9t8fOBjbh1/K8kRyU6Ho5RSJ6XJwgFV9VU8uvZRsgdls3D0QqfDUUopr3SC2wG//OyXlNaW8sQ5TxAYEOh0OEop5ZX2LHrY5iObWf7lchaOXkjOoBynw1FKKZ9osuhBTa4mfvzJjxkUPohbx9/qdDhKKeUzTRY9aPlXy9laupV7Jt1DdEi00+EopZTPdM6ih5TUlPDLz37JtNRpnJ91vtPhqF6uoaGBoqIiamtrnQ5F9RNhYWGkp6cTHBzcpf01WfSQn6z9CQ1NDXx/yve1UKDyqqioiOjoaLKysvTvRZ0yYwxHjx6lqKiIYcO6Vn/Or8NQIjJbRL4SkR0icm8Hbf5DRLaIyGYR+YPH9itEZLt9ucKfcfrbP4v/ydt73ua6cdeREZPhdDiqD6itrWXQoEGaKFS3EBEGDRp0Sj1Vv/UsRCQQeBI4FygC1orISmPMFo82o4D7gOnGmDIRSba3JwA/BAoAA6yz9y3zV7z+UttYy4OrHyQrJourc692OhzVh2iiUN3pVP+e/NmzmAzsMMbsMsbUA8uB+W3aXAc82ZwEjDEl9vbzgVXGmFL7vlXAbD/G6je//uLXFB0rYvHUxYQEhjgdjlKdNmXKFPLz88nIyCApKYn8/Hzy8/PZs2dPpx7ntdde48svv+z088+YMYMNGzZ0er9mP/vZz/jDH/7gvaGDvvWtb7Fr16527yspKWHWrFlERkZy++23d/gYR48e5ZxzzmHUqFGcf/75VFRUdGuM/kwWacB+j9tF9jZPpwGnici/RGS1iMzuxL6IyPUiUigihYcPH+7G0LvHropdLNu0jLnD5zJlyBSnw1GqS9asWcOGDRu4//77WbhwIRs2bGDDhg1kZWV16nG6mixORUNDAy+++CILF/buSgk33ngjP/3pT9u9LyIiggcffJBHH330pI/x4IMPcsEFF7B9+3bOPPNMfvKTn3RrjE4vnQ0CRgGzgMuAX4tInK87G2OeNcYUGGMKkpKS/BRi1xhjWLJ6CeFB4Xy34LtOh6OUX/z1r3/ljDPOYMKECSxcuJDq6moA7r77brKzsxk3bhz33HMP//jHP3jrrbe44447utQrafb73/+e008/ndzcXL73ve+5tz/zzDOcdtppTJkyhWuvvdb9DXzVqlVMnjyZwECrUsLq1asZN24c+fn53HXXXeTn5wOwc+dOzjzzTMaPH8/EiRNZs2YNAO+88w5nnXUW8+bNY/jw4SxevJgXX3yRSZMmMW7cOPfruPzyy7n55puZMmUKI0aM4KOPPuKKK65gzJgxXHPNNe44r7/+egoKCsjJyeH+++93b581axZ/+9vfaGpqOuE1R0VFMX36dMLCTn4KgzfeeIMrrrCmd6+44gpWrFjR2bf3pPy5GqoYGOpxO93e5qkIWGOMaQB2i8g2rORRjJVAPPf9wG+R+sGbu95k7aG1/O/U/yUxPNHpcFQf9uM/b2bLgcpufczs1Bh+eNGpVRAoKSnhkUce4d1333V/+3388ce55ppreOutt9i8eTMiQnl5OXFxcVx44YUsWLCAiy++uEvPV1RUxOLFiyksLCQ2NpZvfOMbvPnmm+Tl5fHII4+wfv16IiMjmTVrFpMnTwbgX//6FxMnTnQ/xlVXXcULL7zA5MmTueuuu9zbhwwZwqpVqwgLC+PLL7/kiiuucCeMzz//nK1btxIbG0tWVhY33XQTa9euZenSpTzxxBP87Gc/A6CiooI1a9bwpz/9iYsuuohPPvmEMWPGMGHCBDZt2kRubi6PPPIICQkJNDY2ctZZZ7FgwQKys7MJDAwkKyuLTZs2kZeX16X35+jRozR/aU5LS+PgwYNdepyO+LNnsRYYJSLDRCQEuBRY2abNCuykICKJWMNSu4C3gfNEJF5E4oHz7G19QkVdBT8r/Bnjksax4LQFToejlF98/PHHbNmyhWnTppGfn89LL73Enj17SEhIICAggOuuu47XX3+dyMjIbnm+NWvWcPbZZ5OYmEhwcDDf/va3+eijj9zb4+PjCQkJYcGClv+5gwcPuj9Ajxw5Qn19vTuRfPvb33a3q6ur45prriE3N5dLL72ULVvc63CYMmUKKSkphIWFMXz4cM4/3zpO6vTTT2/VQ7rooovc21NTU8nOziYgIIDs7Gx3uz/+8Y9MmDCBCRMmsHXr1lbPk5yczIEDB7rlvfIHv/UsjDGNInIL1od8ILDMGLNZRO4HCo0xK2lJCluAJuBuY8xRABF5ACvhANxvjCn1V6zd7fH1j1NRV8Gz5z5LgDg90qf6ulPtAfiLMYbZs2fzu9/97oT7CgsLWbVqFa+88gpPP/00f//73zt8HM8P8G9+85v84Ac/6LYYw8PDfVouunTpUoYOHcrvf/97GhoaiIqKct8XGhrqvh4QEOC+HRAQQGNj4wntPNt4ttu+fTuPP/44n376KXFxcVx++eWtYqutrSU8PJxXX32VJUuWAPD888+7h8q8GTRoEIcPHyYpKYni4mKGDBni036+8usnmTHmLWPMacaYEcaYB+1tP7ATBcZypzEm2xhzujFmuce+y4wxI+3Lb/0ZZ3faULKBV7a9wnfGfofRCaOdDkcpv5k2bRoffvihexVPdXU127dvp6qqisrKSubOncvPf/5zPvvsMwCio6Opqqo64XFCQkLck+YnSxRTpkzh/fff5+jRozQ2NrJ8+XJmzpzJ5MmTef/99ykvL6ehoYHXXnvNvc/YsWPZsWMHgLtHUlhYCMDy5e6PGyoqKhgyZAgiwgsvvIAx5tTfoDYqKyuJjo4mJiaGgwcP8vbbrQdLtm/fTk5ODgsWLHC/H74mCoB58+bxwgsvAPDCCy8wf37bxaenRr/2dqNGVyMPrH6AlIgUbsq/yelwlPKrlJQUfvOb37Bw4ULy8vKYNm0a27Zto6Kigjlz5pCXl8fMmTN57LHHALjssst46KGHujzBnZ6ezgMPPMCsWbPIz89n6tSpzJkzh4yMDO6++24mTZrEjBkzGD58OLGxsQBceOGFfPjhh+7HWLZsGVdddRXjx4+ntrbW3e6WW27hueeeIy8vj927d7fqGXSXCRMmkJ2dzZgxY1i0aBHTp7ecSvnAgQPExsbS0UKd9PR0/ud//off/OY3pKen89VXXwHWHEzzsuLvfe97/OUvf2HUqFF89NFH3H13N5+B0xjTLy4TJ040Tnt+0/Mm9/lc886ed5wORfVxW7ZscTqEPqWqqsoYY0x9fb254IILzMqVK933XXTRRWbnzp2t2hljzJIlS8ydd97Zs4F24Cc/+Yl5/vnn/f487f1dYU0LeP2M1Z5FNzlUfYgnNzzJzPSZnJ1xttPhKDWg/O///i/jx49n3LhxjB49mrlz57rve/TRR90TxytXriQ/P5/c3Fw++eQT7rvvPqdCbmXQoEFcfvnlTodxUmL8MDbnhIKCAtM8FumE29+/nX8V/4sVF68gLeqE4weV6pStW7cyduxYp8NQ/Ux7f1ciss4YU+BtX+1ZdIMP9n/Au/ve5ca8GzVRKKX6JU0Wp6imoYaH1zzMyLiRLMpZ5HQ4SinlF3o+i1P0zMZnOFB9gOdnP09wQNdOKqKUUr2d9ixOwfay7by4+UUuGXkJE1Mmet9BKaX6KE0WXeQyLpasXkJUSBR3TLzD6XCU8hstUe5/bUuUr127ltzcXEaOHMkdd7T/+WKM4aabbmLkyJGMGzfulN4jX2iy6KI3drzB+pL13DnxTuLD4p0ORym/0RLl/te2RPmNN97Ib3/7W7Zv387mzZtZtWrVCfv8+c9/Zv/+/ezYsYOnnnqKm2++2a8xarLogrLaMpauW8qE5AnMH9m9h9Qr1ZdoiXLrdXRnifL9+/dTW1vLpEmTEBH+8z//s91y42+88QaLFlmLambMmMGhQ4fw53l9dIK7Cx5b9xjV9dUsnrpYCwUq//vrvXDoi+59zMGnwwWPnNJDaIly/5QoP378OEOHtpzdIT09neLitmd3gOLi4nbb+evcPvpJ10mFhwpZsWMFi3IWMSp+lNPhKOUYLVGuJcpVBxqaGliyegmpkancMO4Gp8NRA8Up9gD8xWiJcr+UKE9LS2P//pazShcVFZGWduLBvs3tpk6detJ23UV7Fp3wwpYX2Fmxk+9N+R4RwRFOh6OUo7REeef4WqJ86NChhIaGsnbtWowx/O53v2u33Pi8efN48cUXAfjnP/9JSkqK34agQHsWPiuqKuKZz5/hnIxzmDl0ptPhKOU4zxLl9fX1ADz00EOEh4fzzW9+k7q6OlwuV6sS5TfccANLly5lxYoVnV5N5Vmi3BjDRRddxJw5cwDcJcoTEhIYPXp0qxLlnhPMzSXKg4KCOPPMM1uVKF+wYAHLli1jzpw5fi9RnpmZedIS5U8//TRXXnkltbW1zJ07l/POOw+AJ598ktDQUK699louuugi/vrXvzJixAgiIiLcicNvfClN2xcu/ixR7nK5zH+t+i8z6feTzMFjB/32PEo10xLlnaMlyn2jJcr97N197/KP4n9wc/7NDI4c7HQ4Sqk2tES5/2mJci+qG6qZt2Ie8aHxLJ+7nKAAHblT/qclypU/nEqJcv3k8+LJDU9yuOYwj816TBOFUmrA0mGok9h6dCsvbX2JBactIC8pz+lwlFLKMZosOtDkauKB1Q8QFxrHbRNuczocpZRylCaLDvxp+5/44sgX3FVwF7GhsU6Ho5RSjtJk0Y4jx4/wi3W/YPLgycwdPtf7Dkr1Y1qi3P/alii/9957SU9PJy4u7qT7LVmyhJEjRzJmzBjeeecdv8aoM7btWFq4lNqmWhZPXYyIOB2OUo5qLqj3/PPPU1hYyBNPPNGlx3nttdcICAhgzJgx3RneSTWXKG8+iry3ai5R/vTTTwMwf/58brnlFnJzczvcZ+PGjbz22mts2bKF/fv3M3v2bL766isCAvzTB9CeRRtrDq7hzV1vcnXu1QyLHeZ0OEr1alqi3Hod3VmiHOCMM85g8OCTH9P1xhtvcNlllxESEsKIESPIyMhg3bp1XXpffaE9Cw/1TfUsWb2EodFDufb0a50ORykAHv30Ub4s7d6TBo1JGMM9k+85pcfQEuX+KVGel+fbysvi4mJmzZrlvt1conzSpElden+90Z6Fh2WblrGncg/fn/J9woLCnA5HqV5NS5RrifJuIyKzgceBQOA5Y8wjbe6/Evgp0HxmjyeMMc/Z9zUBzWd82WeMmefPWPdV7uPXG3/N7KzZTE+b7n0HpXrIqfYA/MVoiXK/lCj3la+lzLuL33oWIhIIPAlcAGQDl4lIdjtNXzbG5NuX5zy2H/fY7tdEYYzhwTUPEhIYwt2T7vbnUynVb2iJ8s7xtUS5r+bNm8cf//hH6uvr2blzJ3v37m015Nbd/DkMNRnYYYzZZYypB5YDvfKE1W/veZuPD3zMreNvJTki2elwlOoTPEuU5+XlMW3aNLZt20ZFRQVz5swhLy+PmTNntipR/tBDD3V5gtuzRHl+fj5Tp05lzpw5ZGRkuEuUz5gxg+HDh7cqUf7hhx+6H6O5RPn48eOpra1tVaL8ueeeIy8vj927d/u9RPmiRYtOWqL8zjvvJCsri8rKStLT01myZAkAr7/+untiPC8vj4svvpixY8dy4YUX8tRTT/ltJRTgvxLlwAKsoafm2/+JNczk2eZK4CCwEXgVGOpxXyNQCKwGLu7gOa632xRmZGR0qWRvZV2lmfXyLPMff/4P09jU2KXHUKq7aYnyztES5b7pyyXK/wxkGWPGAauAFzzuyzRWJcRvA78QkRFtdzbGPGuMKTDGFHT1DFF1TXWcnng6P5j6AwIDArv0GEopZ2mJcv/zW4lyETkD+JEx5nz79n0AxpiHO2gfCJQaY06orSEizwNvGmNe7ej5/FWiXCknaIly5Q+nUqLcnz2LtcAoERkmIiHApcBKzwYiMsTj5jxgq709XkRC7euJwHRgC0oppRzht6WzxphGEbkFeBtr6ewyY8xmEbkfa4xsJfDfIjIPa36iFGsOA2As8IyIuLAS2iPGGE0WakAxxmi5GdVtTnUUSc+Up1QvtHv3bqKjoxk0aJAmDHXKjDEcPXqUqqoqhg1rXcZIz5SnVB+Wnp5OUVERhw8fdjoU1U+EhYWRnp7e5f01WSjVCwUHB5/wDVApJzm9dFYppVQfoMlCKaWUV5oslFJKedVvVkOJyGFg7yk8RCJwpJvC6U4aV+doXJ2jcXVOf4wr0xjjtQRGv0kWp0pECn1ZPtbTNK7O0bg6R+PqnIEclw5DKaWU8kqThVJKKa80WbR41ukAOqBxdY7G1TkaV+cM2Lh0zkIppZRX2rNQSinl1YBNFiLyLRHZLCIuEelwFYGIzBaRr0Rkh4jc2wNxJYjIKhHZbv+M76Bdk4hssC8r22vTTfGc9PWLSKiIvGzfv0ZEsvwVSydiulJEDnu8P9f6Oyb7eZeJSImIbOrgfhGR/2fHvVFEJvSSuGaJSIXH+9XxibC7N66hIvK+iGyx/xdva6dNj79nPsbV4++ZiISJyKci8rkd14/baeO//0dfTqfXHy9YZdBHAx8ABR20CQR2AsOBEOBzINvPcf0EuNe+fi/waAftjvXAe+T19QM3Ab+yr18KvNwLYrqSNqfw7aG/qX8DJgCbOrj/QuCvgABTgTW9JK5ZWCcX6+n3awgwwb4eDWxr53fZ4++Zj3H1+HtmvwdR9vVgYA0wtU0bv/0/DtiehTFmqzHmKy/NJgM7jDG7jDH1wHJgvp9Dm0/L6WVfAC728/OdjC+v3zPeV4FzxL81tZ34nfjEGPMR1nlZOjIfeNFYVgNxbU4A5lRcjjDGHDTGrLevV2Gd/CytTbMef898jKvH2e/BMftmsH1pO+nst//HAZssfJQG7Pe4XYT//2hSjDEH7euHgJQO2oWJSKGIrBYRfyUUX16/u40xphGoAAb5KR5fYwL4d3vY4lURGerHeDrDib8nX51hD2/8VURyevrJ7eGS8Vjflj05+p6dJC5w4D0TkUAR2QCUAKuMMR2+X939/9ivS5SLyDvA4Hbu+r4x5o2ejqfZyeLyvGGMMSLS0XK1TGNMsYgMB94TkS+MMTu7O9Y+6s/AH40xdSJyA9Y3rbMdjqk3W4/193RMRC4EVgCjeurJRSQK+BNwuzGmsqee1xsvcTnynhljmoB8EYkDXheRXGNMu3NR3a1fJwtjzDdO8SGKAc9vpen2tlNysrhE5GsRGWKMOWh3t0s6eIxi++cuEfkA69tPdycLX15/c5siEQkCYoGj3RxHp2Iyxng+/3NY80C9gV/+nk6V5wehMeYtEXlKRBKNMX6vgSQiwVgfyC8ZY15rp4kj75m3uJx8z+znLBeR94HZgGey8Nv/ow5DndxaYJSIDBOREKwJI7+tPLKtBK6wr18BnNADEpF4EQm1rycC0wF/nKPcl9fvGe8C4D1jz675ideY2oxpz8Mac+4NVgKL7BU+U4EKjyFHx4jI4OZxbRGZjPW54M+E3/y8AvwG2GqMeayDZj3+nvkSlxPvmYgk2T0KRCQcOBf4sk0z//0/9uRsfm+6AJdgjX/WAV8Db9vbU4G3PNpdiLUaYifW8JW/4xoEvAtsB94BEuztBcBz9vVpwBdYK4G+AK7xYzwnvH7gfmCefT0MeAXYAXwKDO+B98hbTA8Dm+33531gTA/9Tf0ROAg02H9b1wA3Ajfa9wvwpB33F3SwCs+BuG7xeL9WA9N6KK4ZWBO0G4EN9uVCp98zH+Pq8fcMGAd8Zse1CfiBvb1H/h/1CG6llFJe6TCUUkoprzRZKKWU8kqThVJKKa80WSillPJKk4VSSimvNFko1Qkicsx7q5Pu/6p91D0iEiUiz4jIThFZJyIfiMgUEQkRkY/sg6qU6hU0WSjVQ+z6QYHGmF32puewCvyNMsZMBK4CEo1VIPFdYKEzkSp1Ik0WSnWBfUTxT0Vkk4h8ISIL7e0BdumHL8U6H8lbIrLA3u072Efki8gIYAqw2BjjAjDG7DbG/MVuu8Jur1SvoN1cpbrmm0A+kAckAmtF5COs0itZQDaQjFVqZJm9z3Sso6kBcoANxioM155NwCS/RK5UF2jPQqmumYFV2bbJGPM18CHWh/sM4BVjjMsYcwir3EizIcBhXx7cTiL1IhLdzXEr1SWaLJTqOcexaveAVVcoT0QCT9I+FKj1e1RK+UCThVJd8w9goX0ymiSsU5d+CvwL68RLASKSgnX6zWZbgZEAxjr3SCHwY4/qpVkiMse+Pgg4Yoxp6KkXpNTJaLJQqmtex6r++TnwHvA/9rDTn7Aqu24Bfo91kpwKe5+/0Dp5XIt1JsQdIrIJeJ6W85ecZbdXqlfQqrNKdTMRiTLWGdQGYfU2phtjDtnnIHjfvt3RxHbzY7wG3GuM2dYDISvlla6GUqr7vWmfpCYEeMDucWCMOS4iP8Q6T/K+jna2T+q0QhOF6k20Z6GUUsornbNQSinllSYLpZRSXmmyUEop5ZUmC6WUUl5pslBKKeWVJgullFJe/X8cNgobHcGXiwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#accuracy_s1 =np.array(accuracy_s).reshape(len(C_s),len(gamma_s))\n",
    "Otto_SVM_result = pd.read_csv(\"Otto_SVM_result.csv\")\n",
    "accuracy_s1 = Otto_SVM_result['accuracy']\n",
    "\n",
    "C_s = np.logspace(-1, 3, 5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-1, 1, 3)  \n",
    "accuracy_s1 =np.array(accuracy_s1).reshape(len(C_s),len(gamma_s))\n",
    "\n",
    "x_axis = np.log10(C_s)\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    plt.plot(x_axis, np.array(accuracy_s1[:,j]), label = ' Test - log(gamma)' + str(np.log10(gamma)))\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "plt.savefig('RBF_SVM_Otto.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.0\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "### 最佳超参数\n",
    "index = np.unravel_index(np.argmax(accuracy_s1, axis=None), accuracy_s1.shape)\n",
    "Best_C = C_s[ index[0] ]\n",
    "Best_gamma = gamma_s[ index[1] ]\n",
    "\n",
    "print(Best_C)\n",
    "print(Best_gamma)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 找到最佳参数后，用全体训练数据训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# SVC训练SVC，支持概率输出\n",
    "Best_C = 100\n",
    "Best_gamma = 1.0\n",
    "\n",
    "SVC4 =  SVC( C = Best_C, kernel='rbf', gamma = Best_gamma, probability=True)\n",
    "SVC4.fit(X_train, y_train)\n",
    "\n",
    "#保持模型，用于后续测试\n",
    "import cPickle\n",
    "cPickle.dump(SVC4, open(\"Otto_RBF_SVC.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
